﻿const puppeteer = require('puppeteer');
//*****下句puppeteer模拟移动设备 内置插件（无需安装），用于抓取移动端网页
const devices = require('puppeteer/DeviceDescriptors')
const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');
const {promisify}=require('util')
let log = (a) => {
    console.log(a);
};
let startName='cpjs' //需要抓取页面的后缀
const zjPage=startName+'Page';  //该页面数据的json名字

const publicDir=['css','js','img']  //写上要创建的文件夹
fs.access('./public', fs.constants.F_OK, (err) => { //如果没有public目录则创建和创建上面三个目录
    if (err){
        fs.mkdir('./public', err => {
            if (err)throw err;
            publicDir.forEach((item) => {
                fs.mkdir('./public/'+item, (err) => {
                    if (err) throw err;
                })
            })
        });
    }else{
        console.log('已有文件夹');
    }
});
fs.access('./json', fs.constants.F_OK, (err) => { //如果没有public目录则创建和创建上面三个目录
    if (err){
        fs.mkdir('./json', err => {
            if (err)throw err;
            fs.writeFileSync(__dirname+'/json/'+zjPage+'.json','')
        });
    }else{
        console.log('已有文件夹');
    }
});
try{
    fs.accessSync(__dirname+'/txtKeepHtml.txt', fs.constants.F_OK);
    console.log('已有txtKeepHtml.txt');
}catch (err) {
    const writeErr=fs.writeFileSync(__dirname+'/txtKeepHtml.txt','');
   if (writeErr)throw err;
}

// const nav=[ 0, 1, 2, 3, 4, 5 ];  //页数，外部获取传入
const nav=[ 0, 1, 2, 3,4,5];  //页数，外部获取传入

(async () => {
    for (let a = 0; a < nav.length; a++) {  //去掉上一页、1页和最后的下一页
        await startChildHtml(a)
    }
})();
function startChildHtml(a) {
    return new Promise((resolve, reject) => {
        const htmlList=fs.readFileSync(__dirname+'/txtKeepHtml.txt','utf8')
        const listArr=htmlList.split(',')
        let txtKeepHtml=[];
        let zjRouters = [];  //接收路由表名
        let urlArr = [];  //接收每页的静态资源文件地址
        // let htmlArr=[ ...nav,...listArr];
        let htmlArr=[ ...listArr];
        let zjfuwei = 'https://www.xxxx.cn';
        let zjfuweiUrl = 'https://www.xxxx.cn/'+startName;
        (async () => {
            //自定义一个等待事件方法，其实与page.waitFor(time)一样，只是有些地方不方便用page.waitFor(time)
            let sleep = time => new Promise(resolve => {
                setTimeout(resolve, time);
            });
            const browser = await puppeteer.launch({
                headless: true,
                slowMo: 250
            });   //获取浏览器对象
            const page = await browser.newPage();     //实例化一个窗口对象
            await page.emulate(devices['iPhone 6'])   //*****移动端网页只需加上这句
            await page.setDefaultNavigationTimeout(3000000)

            await page.goto(zjfuweiUrl,{waitUntil: 'networkidle2', timeout:3000000});
            await sleep(5000);
            let seveUrl=[];  //用来存所有静态资源文件地址最后才处理下载它们
            await page.click('.mw-page-more');
            await page.on('response',response=>{
                if(response.url().indexOf('GetData')>0){
                    response.text().then(data => {
                        let dataArr=data.match(/\/\/nwzimg\.wezhan\.cn(\/(\w)*){4,5}\.jpg/gi)
                        console.log(dataArr)
                        dataArr.forEach((item)=>{
                            seveUrl.push('https:'+item)
                        })
                        const regData=new RegExp(/\/\/nwzimg\.wezhan\.cn(\/(\w)*){3,4}\//,'gi')
                        let data2=data.replace(regData,'./public/img/');
                        fs.writeFileSync(__dirname + '/json/' + zjPage + '.json', data2+',', {flag: 'a'});
                    });
                }
            })
            await sleep(3000);
            zjRouters = await page.evaluate((htmlArr) => {
                console.dir(htmlArr);
                var $ = window.$;
                var arr = [];
                // $('.mw-list-link').each(function () {
                $('.mw-list-link').each(function () {
                    const aHref=$(this).attr('href');
                    if(aHref&&aHref.indexOf('.html')>0)
                        if(htmlArr.indexOf(aHref)<0&&aHref) {
                            arr.push(aHref);
                        }
                });
                return arr;
            },htmlArr);
            await browser.close();
            txtKeepHtml=await zjRouters.toString();
            await console.log(zjRouters);
            await sleep(2000);
            // await (()=>{ //测试时测试一个页面用
            //     zjRouters=['/sy'];
            // })()

            const browser2 = await puppeteer.launch({
                devtools: true,
                headless: false,
                slowMo: 250
            });   //获取浏览器对象
            const page2 = await browser2.newPage();
            await page2.emulate(devices['iPhone 6'])
            //这个page.on('response'窃听，只要page对象没有close  下面操作页面也会传给它窃听
            await page2.on('response', response => {  //检测请求头url，也就是浏览器的NetWork

               repUrl(urlArr,seveUrl,response)
            });
            await sleep(1000);
            await (async () => {   //await自调函数，封装for遍历等待顺序执行
                for (let i = 0; i < zjRouters.length; i++) {  //里面有await用for别用forEach遍历
                    let id = zjRouters[i];
                    let url = zjfuwei + id;
                    urlArr = [];
                    const idDir = id.split('/');
                    let addDir = '';
                    await (async () => {
                        for (let j = 1; j < idDir.length - 1; j++) {
                            await (async (j) => {
                                return new Promise((resolve, reject) => {
                                    let dir = path.join(__dirname, addDir, idDir[j]);
                                    promisify(fs.access)(dir, fs.constants.F_OK).then(data => {
                                        if (!data) {
                                            console.log('已有目录', dir);
                                        }
                                    }).catch(err => {
                                        if (err) {
                                            fs.mkdir(dir, err => {
                                                if (err) throw err;
                                                addDir = path.join(addDir, idDir[j]);
                                            });
                                        }
                                    });
                                    resolve();
                                });
                            })(j);
                        }
                    })();
                    await page.setDefaultNavigationTimeout(3000000)
                    let urlResponse = await page2.goto(url,{waitUntil: 'networkidle2', timeout:3000000});
                    await sleep(500);
                    await urlResponse.text()  //这个是puppeteer的api获取当前页面整个dom包括所有
                        .then((data) => {
                            //then里如果是一个结果就不需return，如果是一个函数或方法就要return出去
                            return saveFile(urlArr, data);
                        })
                        .then((zjhtml) => {
                            const wPath = id.indexOf('.html') > -1 ? path.join(__dirname, id) : path.join(__dirname, id + '.html');
                            fs.writeFileSync(wPath, zjhtml, 'utf8');
                        })
                        .catch(err => {
                            log(err);
                        });
                }
            })();
            //这里是专门下载静态资源，以及保存在对应类型目录，与saveHtml()处理dom内容href路径要对应
            await (async ()=>{
                for (let i = 0; i < seveUrl.length; i++) {
                    const item = seveUrl[i];
                    const mod = item.indexOf('https') > -1 ? https : http;
                    let pname = path.basename(item);
                    let typePath = path.extname(item).replace('.', '');
                    if (typePath === 'png' || typePath === 'jpg') {
                        typePath = 'img';
                    }
                    const dirPath = path.join('public', typePath, pname);
                    const filePath = path.join(__dirname, dirPath);
                    await mod.get(item, (res) => {   //请求一个个网络地址，取得res数据对象
                        if (res.statusCode !== 200) log(item+'这里断了线')
                        res.pipe(fs.createWriteStream(filePath)) //用流方式写入
                            .on('close', () => {
                                log(filePath);
                            });
                    });
                    await sleep(3000)
                }
            })();
            await browser2.close();
            await fs.writeFile(__dirname+'/txtKeepHtml.txt',txtKeepHtml,{flag:'a'}, (err) => {
                if(err)throw err
            })
            await resolve();
        })();

    })
}
//这里处理网页dom内容(将href网站换成本地public目录)，写成文件分工下一步做
function saveFile(urlArr, zjhtml) {
    return new Promise(async (resolve, reject) => {
        if(!zjhtml||urlArr.length<1){
            reject('没有传资源链接或html内容没传来');
            return
        }
        for (let i = 0; i < urlArr.length; i++) {  //urlArr是读取静态资源网络地址，集成的数组
            let item = urlArr[i];
            //下面将网络资源路径https://www.xxxx.cn/ab/626525.png转为本
            // 地 ./public/xxtype/文件名和后缀
            let pname = path.basename(item);
            let typePath = path.extname(item).replace('.', '');
            if (typePath === 'png' || typePath === 'jpg') {
                typePath = 'img';
            }
            const dirPath = path.join('public', typePath, pname);
            let dirSpl = dirPath.split("\\").join('/');
            if (zjhtml.indexOf(item) > -1) {  //如果html代码匹配准确网络地址直接替换
                let reg = new RegExp(item, 'g');  //替换只有全局g正则才能替换所有，否则只能替换一个
                zjhtml = zjhtml.replace(reg, '../' + dirSpl);
            } else {   //否则html代码可能给改了//www.zjfuwei就把网络地址变成//开头
                let splUrl = item.split('//');
                let reg = new RegExp('//' + splUrl[1], 'g');
                zjhtml = zjhtml.replace(reg, '../' + dirSpl);
            }
        }
        resolve(zjhtml);
    });
}
function repUrl(urlArr,seveUrl,response) {  //下句是只要后缀为jpg...就,且之前没有获取过的
    if (response.url().indexOf('.jpg') > -1 || response.url().indexOf('.png') > -1
        || response.url().indexOf('.js') > -1 || response.url().indexOf('.css') > -1) {
        if (response.url().indexOf('?') > -1) {
            const spl = response.url().split('?');
            urlArr.push(spl[0]);
            if (seveUrl.indexOf(spl[0]) < 0)
                seveUrl.push(spl[0]);
        } else {
            urlArr.push(response.url());
            if (seveUrl.indexOf(response.url()) < 0)
                seveUrl.push(response.url());
        }
    }
}

